{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2d124d22-de73-436b-86cd-9b162b469be8",
   "metadata": {
    "id": "2d124d22-de73-436b-86cd-9b162b469be8"
   },
   "outputs": [],
   "source": [
    "%pip install langchain_community\n",
    "%pip install langchain_experimental\n",
    "%pip install langchain-openai\n",
    "%pip install langchainhub\n",
    "%pip install chromadb\n",
    "%pip install langchain\n",
    "%pip install beautifulsoup4\n",
    "%pip install python-dotenv\n",
    "\n",
    "# new\n",
    "%pip install gensim --user\n",
    "%pip install transformers\n",
    "%pip install torch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "f884314f-870c-4bfb-b6c1-a5b4801ec172",
   "metadata": {
    "executionInfo": {
     "elapsed": 5391,
     "status": "ok",
     "timestamp": 1714991832985,
     "user": {
      "displayName": "",
      "userId": ""
     },
     "user_tz": 240
    },
    "id": "f884314f-870c-4bfb-b6c1-a5b4801ec172"
   },
   "outputs": [],
   "source": [
    "import os\n",
    "from langchain_community.document_loaders import WebBaseLoader\n",
    "import bs4\n",
    "import openai\n",
    "from langchain_openai import ChatOpenAI, OpenAIEmbeddings\n",
    "from langchain import hub\n",
    "from langchain_core.output_parsers import StrOutputParser\n",
    "from langchain_core.runnables import RunnablePassthrough\n",
    "import chromadb\n",
    "from langchain_community.vectorstores import Chroma\n",
    "from langchain_experimental.text_splitter import SemanticChunker\n",
    "from langchain_core.runnables import RunnableParallel\n",
    "from dotenv import load_dotenv, find_dotenv\n",
    "from langchain_core.prompts import PromptTemplate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "eba3468a-d7c2-4a79-8df2-c335542950f2",
   "metadata": {
    "executionInfo": {
     "elapsed": 4,
     "status": "ok",
     "timestamp": 1714991832985,
     "user": {
      "displayName": "",
      "userId": ""
     },
     "user_tz": 240
    },
    "id": "eba3468a-d7c2-4a79-8df2-c335542950f2"
   },
   "outputs": [],
   "source": [
    "# variables\n",
    "_ = load_dotenv(dotenv_path='env.txt')\n",
    "os.environ['OPENAI_API_KEY'] = os.getenv('OPENAI_API_KEY')\n",
    "openai.api_key = os.environ['OPENAI_API_KEY']\n",
    "embedding_function = OpenAIEmbeddings()\n",
    "llm = ChatOpenAI(model_name=\"gpt-4o-mini\")\n",
    "user_query = \"What are the advantages of using RAG?\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "1a025b63-125d-4e2b-b092-c863d7ffff9b",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 1085,
     "status": "ok",
     "timestamp": 1714991834067,
     "user": {
      "displayName": "",
      "userId": ""
     },
     "user_tz": 240
    },
    "id": "1a025b63-125d-4e2b-b092-c863d7ffff9b",
    "outputId": "6aa2a750-c7d5-495c-fbd2-583ec1087153"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "User question embedding (first 5 dimensions): [-0.006319054113595048, -0.0023517232115089787, 0.015498643243434815, -0.02267445873596028, 0.017820641897159206]\n"
     ]
    }
   ],
   "source": [
    "# Obtain embedding for user query\n",
    "question_embedding = embedding_function.embed_query(user_query)\n",
    "first_5_numbers = question_embedding[:5]\n",
    "print(f\"User question embedding (first 5 dimensions): {first_5_numbers}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "4a8331bd-b531-4bb3-82d6-0bd300d941fa",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 7,
     "status": "ok",
     "timestamp": 1714991834067,
     "user": {
      "displayName": "",
      "userId": ""
     },
     "user_tz": 240
    },
    "id": "4a8331bd-b531-4bb3-82d6-0bd300d941fa",
    "outputId": "54c42e48-9f98-4802-b2ca-d6554c2f319c"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Embedding size: 1536\n"
     ]
    }
   ],
   "source": [
    "# Obtain the size of the user query embedding\n",
    "embedding_size = len(question_embedding)\n",
    "print(f\"Embedding size: {embedding_size}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d3ad428a-3eb6-40ec-a1a5-62565ead1e5b",
   "metadata": {
    "id": "d3ad428a-3eb6-40ec-a1a5-62565ead1e5b"
   },
   "outputs": [],
   "source": [
    "#### INDEXING ####"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "98ccda2c-0f4c-41c5-804d-2227cdf35aa7",
   "metadata": {
    "executionInfo": {
     "elapsed": 365,
     "status": "ok",
     "timestamp": 1714991834427,
     "user": {
      "displayName": "",
      "userId": ""
     },
     "user_tz": 240
    },
    "id": "98ccda2c-0f4c-41c5-804d-2227cdf35aa7"
   },
   "outputs": [],
   "source": [
    "# Load Documents\n",
    "loader = WebBaseLoader(\n",
    "    web_paths=(\"https://kbourne.github.io/chapter1.html\",),\n",
    "    bs_kwargs=dict(\n",
    "        parse_only=bs4.SoupStrainer(\n",
    "            class_=(\"post-content\", \"post-title\", \"post-header\")\n",
    "        )\n",
    "    ),\n",
    ")\n",
    "docs = loader.load()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "927a4c65-aa05-486c-8295-2f99673e7c20",
   "metadata": {
    "executionInfo": {
     "elapsed": 2625,
     "status": "ok",
     "timestamp": 1714991839129,
     "user": {
      "displayName": "",
      "userId": ""
     },
     "user_tz": 240
    },
    "id": "927a4c65-aa05-486c-8295-2f99673e7c20"
   },
   "outputs": [],
   "source": [
    "# Split\n",
    "text_splitter = SemanticChunker(embedding_function)\n",
    "splits = text_splitter.split_documents(docs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "5fecbaaa-df9a-47cf-909f-e1f327374efc",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 647,
     "status": "ok",
     "timestamp": 1714991839774,
     "user": {
      "displayName": "",
      "userId": ""
     },
     "user_tz": 240
    },
    "id": "5fecbaaa-df9a-47cf-909f-e1f327374efc",
    "outputId": "82d5c0f7-30b3-48b2-c339-4a16f4ee2895"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Word\t\tTF\t\tIDF\n",
      "----\t\t--\t\t---\n",
      "000         \t0.16\t\t2.95\n",
      "1024        \t0.04\t\t2.95\n",
      "123         \t0.02\t\t2.95\n",
      "13          \t0.04\t\t2.95\n",
      "15          \t0.01\t\t2.95\n",
      "16          \t0.07\t\t2.95\n",
      "192         \t0.06\t\t2.95\n",
      "1m          \t0.08\t\t2.95\n",
      "200         \t0.08\t\t2.95\n",
      "2024        \t0.01\t\t2.95\n"
     ]
    }
   ],
   "source": [
    "# USING TF-IDF\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn.metrics.pairwise import cosine_similarity\n",
    "\n",
    "# Extract the text content from the splits\n",
    "tfidf_documents = [split.page_content for split in splits]\n",
    "\n",
    "# Create a TF-IDF vectorizer\n",
    "tfidf_vectorizer = TfidfVectorizer()\n",
    "\n",
    "# Generate TF-IDF matrix\n",
    "tfidf_matrix = tfidf_vectorizer.fit_transform(tfidf_documents)\n",
    "\n",
    "# Get the vocabulary, term frequencies, and corresponding IDF values\n",
    "vocab = tfidf_vectorizer.get_feature_names_out()\n",
    "tf_values = tfidf_matrix.toarray()\n",
    "idf_values = tfidf_vectorizer.idf_\n",
    "\n",
    "# Create a list of tuples containing word, TF, and IDF values\n",
    "word_stats = list(zip(vocab, tf_values.sum(axis=0), idf_values))\n",
    "\n",
    "# Sort the list by IDF values in descending order\n",
    "word_stats.sort(key=lambda x: x[2], reverse=True)\n",
    "\n",
    "# Print the grid of top 10 words, TF, and IDF values\n",
    "print(\"Word\\t\\tTF\\t\\tIDF\")\n",
    "print(\"----\\t\\t--\\t\\t---\")\n",
    "for word, tf, idf in word_stats[:10]:\n",
    "    print(f\"{word:<12}\\t{tf:.2f}\\t\\t{idf:.2f}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "8c4edd43-5b4b-4dd1-832a-174b7efdfdb4",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 3,
     "status": "ok",
     "timestamp": 1714991839774,
     "user": {
      "displayName": "",
      "userId": ""
     },
     "user_tz": 240
    },
    "id": "8c4edd43-5b4b-4dd1-832a-174b7efdfdb4",
    "outputId": "ded08d3a-1d2d-4201-c52b-3ca529106815"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "TF-IDF Top Document:\n",
      " Can you imagine what you could do with all of the benefits mentioned above, but combined with all of the data within your company, about everything your company has ever done, about your customers and all of their interactions, or about all of your products and services combined with a knowledge of what a specific customer’s needs are? You do not have to imagine it, that is what RAG does! Even smaller companies are not able to access much of their internal data resources very effectively. Larger companies are swimming in petabytes of data that is not readily accessible or is not being fully utilized. Prior to RAG, most of the services you saw that connected customers or employees with the data resources of the company were really just scratching the surface of what is possible compared to if they could access ALL of the data in the company. With the advent of RAG and generative AI in general, corporations are on the precipice of something really, really big. Comparing RAG with Model Fine-Tuning#\n",
      "Established Large Language Models (LLM), what we call the foundation models, can learn in two ways:\n",
      " Fine-tuning - With fine-tuning, you are adjusting the weights and/or biases that define the model's intelligence based on new training data. This directly impacts the model, permanently changing how it will interact with new inputs. Input/Prompts - This is where you actually \"use\" the model, using the prompt/input to introduce new knowledge that the LLM can act upon. Why not use fine-tuning in all situations?\n"
     ]
    }
   ],
   "source": [
    "# TD-IDF scoring for user query\n",
    "# User query embedding\n",
    "tfidf_user_query = [user_query]\n",
    "new_tfidf_matrix = tfidf_vectorizer.transform(tfidf_user_query)\n",
    "\n",
    "# Calculate cosine similarity between the new content and the original documents\n",
    "tfidf_similarity_scores = cosine_similarity(new_tfidf_matrix, tfidf_matrix)\n",
    "\n",
    "# Find the index of the document with the highest similarity score\n",
    "tfidf_top_doc_index = tfidf_similarity_scores.argmax()\n",
    "\n",
    "# Print the text of the top document\n",
    "print(\"TF-IDF Top Document:\\n\", tfidf_documents[tfidf_top_doc_index])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "72de19a4-f767-434d-b0fa-7efd08c2d1dc",
   "metadata": {
    "executionInfo": {
     "elapsed": 264,
     "status": "ok",
     "timestamp": 1714991842798,
     "user": {
      "displayName": "",
      "userId": ""
     },
     "user_tz": 240
    },
    "id": "72de19a4-f767-434d-b0fa-7efd08c2d1dc"
   },
   "outputs": [],
   "source": [
    "# CREATING AND SAVING DOC2VEC MODEL\n",
    "from gensim.models.doc2vec import Doc2Vec, TaggedDocument\n",
    "from sklearn.metrics.pairwise import cosine_similarity\n",
    "\n",
    "# Extract the text content from the splits\n",
    "doc2vec_documents = [split.page_content for split in splits]\n",
    "\n",
    "# Tokenize the documents\n",
    "doc2vec_tokenized_documents = [doc.lower().split() for doc in doc2vec_documents]\n",
    "\n",
    "# Create tagged documents for Doc2Vec\n",
    "doc2vec_tagged_documents = [TaggedDocument(words=doc, tags=[str(i)]) for i, doc in enumerate(doc2vec_tokenized_documents)]\n",
    "\n",
    "# Train the Doc2Vec model\n",
    "# Use this version first.\n",
    "doc2vec_model = Doc2Vec(doc2vec_tagged_documents, vector_size=100, window=5, min_count=1, workers=4)\n",
    "\n",
    "# After running the previous version of model, comment the previous line out and uncomment this one. Try it with 1536D vectors.\n",
    "# doc2vec_model = Doc2Vec(doc2vec_tagged_documents, vector_size=1536, window=5, min_count=1, workers=4)\n",
    "\n",
    "# Save the trained model to a file\n",
    "doc2vec_model.save(\"doc2vec_model.bin\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "954ce29e-f71b-4efa-8d44-beda4b9bf11f",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 146,
     "status": "ok",
     "timestamp": 1714991846416,
     "user": {
      "displayName": "",
      "userId": ""
     },
     "user_tz": 240
    },
    "id": "954ce29e-f71b-4efa-8d44-beda4b9bf11f",
    "outputId": "d1e7e5d2-0ed7-428d-bbf4-78215c86f75f"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Doc2Vec Top Document:\n",
      " Both! All vector databases are vector stores, but not all vector stores are vector databases. Ok, while you get out your chalkboard to draw the Vinn diagram, I continue to explain this statement. There are ways to store vectors that are not full databases. They are simply storage devices for vectors. So to encompass all possible ways to store vectors, LangChain calls them all vector stores. Let’s do the same! Just know that not all of the “vector stores” that LangChain connects with are officially considered vector databases, but in general, most of them are and many people refer to all of them as vector databases. Phew, glad we cleared that up!\n"
     ]
    }
   ],
   "source": [
    "# USING DOC2VEC SAVED MODEL\n",
    "\n",
    "# Load the saved model\n",
    "loaded_doc2vec_model = Doc2Vec.load(\"doc2vec_model.bin\")\n",
    "\n",
    "# Calculate the document vectors\n",
    "doc2vec_document_vectors = [loaded_doc2vec_model.dv[str(i)] for i in range(len(doc2vec_documents))]\n",
    "\n",
    "# User query for embedding\n",
    "doc2vec_user_query = [user_query]\n",
    "\n",
    "# Tokenize the new content\n",
    "doc2vec_tokenized_user_query = [content.lower().split() for content in doc2vec_user_query]\n",
    "\n",
    "# Infer the vector for the new content\n",
    "doc2vec_user_query_vector = loaded_doc2vec_model.infer_vector(doc2vec_tokenized_user_query[0])\n",
    "\n",
    "# Calculate cosine similarity between the new content vector and the document vectors\n",
    "doc2vec_similarity_scores = cosine_similarity([doc2vec_user_query_vector], doc2vec_document_vectors)\n",
    "\n",
    "# Find the index of the document with the highest similarity score\n",
    "doc2vec_top_doc_index = doc2vec_similarity_scores.argmax()\n",
    "\n",
    "# Print the text of the top document\n",
    "print(\"\\nDoc2Vec Top Document:\\n\", doc2vec_documents[doc2vec_top_doc_index])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "lNj02nuvbRkQ",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 10788,
     "status": "ok",
     "timestamp": 1714991866519,
     "user": {
      "displayName": "",
      "userId": ""
     },
     "user_tz": 240
    },
    "id": "lNj02nuvbRkQ",
    "outputId": "00bbc2df-9ea2-4134-885e-80c3dd82d4df"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.10/dist-packages/huggingface_hub/utils/_token.py:88: UserWarning: \n",
      "The secret `HF_TOKEN` does not exist in your Colab secrets.\n",
      "To authenticate with the Hugging Face Hub, create a token in your settings tab (https://huggingface.co/settings/tokens), set it as secret in your Google Colab and restart your session.\n",
      "You will be able to reuse this secret in all of your notebooks.\n",
      "Please note that authentication is recommended but still optional to access public models or datasets.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Vector size of BERT (base-uncased) embeddings: 768\n",
      "\n",
      "BERT Top Document:\n",
      " Or if you are developing in a legal field, you may want it to sound more like a lawyer. Vector Store or Vector Database?\n"
     ]
    }
   ],
   "source": [
    "# USING BERT\n",
    "from transformers import BertTokenizer, BertModel\n",
    "import torch\n",
    "from sklearn.metrics.pairwise import cosine_similarity\n",
    "\n",
    "# Extract the text content from the splits\n",
    "bert_documents = [split.page_content for split in splits]\n",
    "\n",
    "# Load BERT tokenizer and model\n",
    "bert_tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')\n",
    "bert_model = BertModel.from_pretrained('bert-base-uncased')\n",
    "\n",
    "# Get the vector size of the BERT embeddings\n",
    "bert_vector_size = bert_model.config.hidden_size\n",
    "print(f\"Vector size of BERT (base-uncased) embeddings: {bert_vector_size}\\n\")\n",
    "\n",
    "# Tokenize the documents\n",
    "bert_tokenized_documents = [bert_tokenizer(doc, return_tensors='pt', max_length=512, truncation=True) for doc in bert_documents]\n",
    "\n",
    "# Calculate the document embeddings\n",
    "bert_document_embeddings = []\n",
    "with torch.no_grad():\n",
    "    for doc in bert_tokenized_documents:\n",
    "        bert_outputs = bert_model(**doc)\n",
    "        bert_doc_embedding = bert_outputs.last_hidden_state[0, 0, :].numpy()\n",
    "        bert_document_embeddings.append(bert_doc_embedding)\n",
    "\n",
    "# New content (question) for embedding\n",
    "bert_user_query = [user_query]\n",
    "\n",
    "# Tokenize the new content\n",
    "bert_tokenized_user_query = bert_tokenizer(bert_user_query[0], return_tensors='pt', max_length=512, truncation=True)\n",
    "\n",
    "# Calculate the embedding for the new content\n",
    "bert_user_query_embedding = []\n",
    "with torch.no_grad():\n",
    "    bert_outputs = bert_model(**bert_tokenized_user_query)\n",
    "    bert_user_query_embedding = bert_outputs.last_hidden_state[0, 0, :].numpy()\n",
    "\n",
    "# Calculate cosine similarity between the new content embedding and the document embeddings\n",
    "bert_similarity_scores = cosine_similarity([bert_user_query_embedding], bert_document_embeddings)\n",
    "\n",
    "# Find the index of the document with the highest similarity score\n",
    "bert_top_doc_index = bert_similarity_scores.argmax()\n",
    "\n",
    "# Print the text of the top document\n",
    "print(\"BERT Top Document:\\n\", bert_documents[bert_top_doc_index])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "6b13568c-d633-464d-8c43-0d55f34cc8c1",
   "metadata": {
    "executionInfo": {
     "elapsed": 1218,
     "status": "ok",
     "timestamp": 1714991894573,
     "user": {
      "displayName": "",
      "userId": ""
     },
     "user_tz": 240
    },
    "id": "6b13568c-d633-464d-8c43-0d55f34cc8c1"
   },
   "outputs": [],
   "source": [
    "# Embed\n",
    "vectorstore = Chroma.from_documents(documents=splits,\n",
    "                                    embedding=embedding_function)\n",
    "\n",
    "retriever = vectorstore.as_retriever()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "101e5aef-4c7f-4302-a2e6-ce1abe70e02b",
   "metadata": {
    "id": "101e5aef-4c7f-4302-a2e6-ce1abe70e02b"
   },
   "outputs": [],
   "source": [
    "# Retrieve the first result using the new content\n",
    "result = retriever.get_relevant_documents(user_query)[0]\n",
    "\n",
    "# Print the retrieved document\n",
    "print(\"\\nRetrieved Document:\\n\", result.page_content)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6ce8df01-925b-45b5-8fb8-17b5c40c581f",
   "metadata": {
    "id": "6ce8df01-925b-45b5-8fb8-17b5c40c581f"
   },
   "outputs": [],
   "source": [
    "#### RETRIEVAL and GENERATION ####"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fac053d8-b871-4b50-b04e-28dec9fb3b0f",
   "metadata": {
    "id": "fac053d8-b871-4b50-b04e-28dec9fb3b0f"
   },
   "outputs": [],
   "source": [
    "# Prompt\n",
    "prompt = hub.pull(\"jclemens24/rag-prompt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5ef30632-13dd-4a34-af33-cb8fab94f169",
   "metadata": {
    "id": "5ef30632-13dd-4a34-af33-cb8fab94f169"
   },
   "outputs": [],
   "source": [
    "# Relevance check prompt\n",
    "relevance_prompt_template = PromptTemplate.from_template(\n",
    "    \"\"\"\n",
    "    Given the following question and retrieved context, determine if the context is relevant to the question.\n",
    "    Provide a score from 1 to 5, where 1 is not at all relevant and 5 is highly relevant.\n",
    "    Return ONLY the numeric score, without any additional text or explanation.\n",
    "\n",
    "    Question: {question}\n",
    "    Retrieved Context: {retrieved_context}\n",
    "\n",
    "    Relevance Score:\"\"\"\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e8975479-b3e3-481d-ad7b-08b4eb3faaef",
   "metadata": {
    "id": "e8975479-b3e3-481d-ad7b-08b4eb3faaef"
   },
   "outputs": [],
   "source": [
    "# Post-processing\n",
    "def format_docs(docs):\n",
    "    return \"\\n\\n\".join(doc.page_content for doc in docs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fd9db713-f705-4b65-800e-2c4e3d0e4ef4",
   "metadata": {
    "id": "fd9db713-f705-4b65-800e-2c4e3d0e4ef4"
   },
   "outputs": [],
   "source": [
    "def extract_score(llm_output):\n",
    "    try:\n",
    "        score = float(llm_output.strip())\n",
    "        return score\n",
    "    except ValueError:\n",
    "        return 0\n",
    "\n",
    "# Chain it all together with LangChain\n",
    "def conditional_answer(x):\n",
    "    relevance_score = extract_score(x['relevance_score'])\n",
    "    if relevance_score < 4:\n",
    "        return \"I don't know.\"\n",
    "    else:\n",
    "        return x['answer']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cc257a4c-eb01-4819-b03b-b9142e377ea6",
   "metadata": {},
   "outputs": [],
   "source": [
    "rag_chain_from_docs = (\n",
    "    RunnablePassthrough.assign(context=(lambda x: format_docs(x[\"context\"])))\n",
    "    | RunnableParallel(\n",
    "        {\"relevance_score\": (\n",
    "            RunnablePassthrough()\n",
    "            | (lambda x: relevance_prompt_template.format(question=x['question'], retrieved_context=x['context']))\n",
    "            | llm\n",
    "            | StrOutputParser()\n",
    "        ), \"answer\": (\n",
    "            RunnablePassthrough()\n",
    "            | prompt\n",
    "            | llm\n",
    "            | StrOutputParser()\n",
    "        )}\n",
    "    )\n",
    "    | RunnablePassthrough().assign(final_answer=conditional_answer)\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "dc5c2ab0-9191-40f7-abf2-681f1c751429",
   "metadata": {
    "id": "dc5c2ab0-9191-40f7-abf2-681f1c751429"
   },
   "outputs": [],
   "source": [
    "rag_chain_with_source = RunnableParallel(\n",
    "    {\"context\": retriever, \"question\": RunnablePassthrough()}\n",
    ").assign(answer=rag_chain_from_docs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8b30177a-f9ab-45e4-812d-33b0f97325bd",
   "metadata": {
    "id": "8b30177a-f9ab-45e4-812d-33b0f97325bd",
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# Question - relevant question\n",
    "result = rag_chain_with_source.invoke(user_query)\n",
    "relevance_score = result['answer']['relevance_score']\n",
    "final_answer = result['answer']['final_answer']\n",
    "\n",
    "print(f\"Relevance Score: {relevance_score}\")\n",
    "print(f\"Final Answer:\\n{final_answer}\")"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "name": "CHAPTER7-VECTORIZERS.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
